# /usr/bin/env python
# -*- coding: UTF-8 -*-
import requests
from bs4 import BeautifulSoup
from crawler import config
from crawler.db.ImageInfoDBOperator import DBOperator, ImageInfo
from logs.Logger import Logger
import time
import sys
from Common import img_download
from requests.exceptions import SSLError
import urllib3
import re
import Common
from date_manage.date_manage import TimeHelper

prefix = 'https://movie.douban.com/subject/'
postfix_stage = '/photos?type=S&start={page}&sortby=like&size=a&subtype=a'
postfix_poster = '/photos?type=R&start={page}&sortby=like&size=a&subtype=a'
postfix_wallpaper = '/photos?type=W&start={page}&sortby=like&size=a&subtype=a'

page_size = 30

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}
# 日志初始化
log = Logger("D:/workspace/logs/douban_crawler.log", level='info')
soap = ''
def url_type(pic_type, id, start='0'):
    url = ''
    if pic_type == 'stage':
        url = prefix + id + postfix_stage
        url = url.replace('{page}', start)
    if pic_type == 'poster':
        url = prefix + id + postfix_poster
        url = url.replace('{page}', start)
    if pic_type == 'wallpaper':
        url = prefix + id + postfix_wallpaper
        url = url.replace('{page}', start)
    return url

def extract_number(s):
    pattern = r'共(\d+)张'
    match = re.search(pattern, s)
    if match:
        return int(match.group(1))
    else:
        return 0

def page_compute(pic_numbers):
    return pic_numbers//page_size + 1

def get_pager(pic_type, id):
    url = url_type(pic_type, id)
    soap = Common.getContextAsSelenium(url)
    pager = soap.find('div', {'class': 'paginator'})
    if pager is not None:
        pic_numbers = extract_number(pager.text)
        page_cnt = page_compute(pic_numbers)
    else:
        page_cnt = 1
    return page_cnt

def get_title(id):
    url = prefix + id
    soap = Common.getContextAsSelenium(url)
    title = soap.find('span', {'property': 'v:itemreviewed'})
    return title.text

def get_type_name(type):
    if type == 'stage':
        return '剧照'
    elif type == 'poster':
        return '海报'
    else:
        return '壁纸'

def get_pic_url(id):
    pic_types = ['stage', 'poster', 'wallpaper']
    url_infos = []
    pic_page_links = []
    pic_links = []
    # title = get_title(id)
    title = '诺博士'
    print(title)
    for pic_type in pic_types:
        i = 1
        page_cnt = get_pager(pic_type, id)
        for i in range(1, page_cnt+1):
            page_index = (i - 1)*30
            url = url_type(pic_type, id, str(page_index))
            url_infos.append((pic_type,url))
    for url_info in url_infos:
        print(url_info)
        soap = Common.getContextAsSelenium(url_info[1])
        ul_soup = soap.find('ul', {'class': 'poster-col3 clearfix'})
        pic_ahrefs =ul_soup.find_all('a')
        for pic_page_ahref in pic_ahrefs:
            ahref = pic_page_ahref.get('href')
            pic_page_links.append((url_info[0], ahref))
    print(pic_page_links)
    for pic_page_link in pic_page_links:
        print(pic_page_link)
        soap = Common.getContextAsSelenium(pic_page_link[1])
        img_div = soap.find('div', {'class': 'photo-wp'})
        link = img_div.find('img')['src']
        pic_links.append((pic_page_link[0],link))
    print(pic_links)
    for pic_link in pic_links:
        pic_type = pic_link[0]
        pic_type_name = get_type_name(pic_type)
        name = title + '/' + pic_type_name
        Common.img_download(pic_link[1],name,None, prefix + id, log)

def get_all_pages(id, stages, posters, wallpapers):
    page_links = []
    for i in range(1, stages + 1):
        page_index = (i - 1) * 30
        url = url_type('stage', id, str(page_index))
        page_links.append(('stage', url))
    for i in range(1, posters + 1):
        page_index = (i - 1) * 30
        url = url_type('poster', id, str(page_index))
        page_links.append(('poster', url))
    for i in range(1, wallpapers + 1):
        page_index = (i - 1) * 30
        url = url_type('wallpaper', id, str(page_index))
        page_links.append(('wallpaper', url))
    for page_link in page_links:
        print("('"+page_link[0]+"','" + page_link[1]+"'),")
    return page_links

def optimized_pic_url(page_links,title):
    image_url_info = []
    for page_link in page_links:
        print(page_link)
        soap = Common.getContextAsSelenium(page_link[1])
        ul_soup = soap.find('ul', {'class': 'poster-col3 clearfix'})
        images = ul_soup.find_all('li')
        for image in images:
            image_url = image.find('img')['src'].replace('/m/', '/1/')
            print((page_link[0], image_url))
            image_url_info.append((page_link[0], image_url))
    for pic_link in image_url_info:
        pic_type = pic_link[0]
        pic_type_name = get_type_name(pic_type)
        name = title + '/' + pic_type_name
        Common.img_download(pic_link[1], name, None, prefix + id, log)


imagelinks = [
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2446168296.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2446168238.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2446168199.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2446168187.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2533483141.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2454467610.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2447679403.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535446918.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2535446911.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535446909.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535446908.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535446905.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2535446901.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2535446897.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2535446893.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535446888.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535446884.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535446880.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535446876.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2535446867.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2535446863.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535446860.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535358868.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535358864.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2535358863.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2535358861.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535358845.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535358839.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2535358832.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535358829.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535358824.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535358819.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535358818.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535274508.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535274505.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2535274502.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535274499.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535274495.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535274490.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535274489.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2535274481.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2535274478.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2535274474.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2535274472.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2534464950.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2534464945.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2534464941.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2534354370.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2534354369.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2534354368.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533907586.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533907585.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533907584.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533907582.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2533907579.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533907577.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533907573.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2533907569.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533907565.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2533835091.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2533835088.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533835087.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533835084.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2533835081.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533835077.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2533835070.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533835066.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533716036.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533716023.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533714375.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533712025.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2533483199.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2533483190.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533483186.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2533483178.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2533483169.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533483164.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533483163.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2533483161.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533483157.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2533483153.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2533483146.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2515054671.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2515054669.webp'),
('stage', 'https://img9.doubanio.com/view/photo/1/public/p2515054665.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2515054663.webp'),
('stage', 'https://img3.doubanio.com/view/photo/1/public/p2454467613.webp'),
('stage', 'https://img2.doubanio.com/view/photo/1/public/p2454467611.webp'),
('stage', 'https://img1.doubanio.com/view/photo/1/public/p2454467608.webp'),
]
def image_download(imagelinks, title, id):
    links = []
    name = ''
    th = TimeHelper()
    current_date = th.getCurrentTimeStr('%Y%m%d%H%M%S')
    for pic_link in imagelinks:
        pic_type = pic_link[0]
        pic_type_name = get_type_name(pic_type)
        name = title + '/' + pic_type_name
        links.append(pic_link[1])
    Common.img_download(links, name, current_date, prefix + id, log)

page_links = [
('stage','https://movie.douban.com/subject/4864908/photos?type=S&start=1440&sortby=like&size=a&subtype=a'),
('stage','https://movie.douban.com/subject/4864908/photos?type=S&start=1470&sortby=like&size=a&subtype=a'),
('stage','https://movie.douban.com/subject/4864908/photos?type=S&start=1500&sortby=like&size=a&subtype=a'),
('poster','https://movie.douban.com/subject/4864908/photos?type=R&start=0&sortby=like&size=a&subtype=a'),
('poster','https://movie.douban.com/subject/4864908/photos?type=R&start=30&sortby=like&size=a&subtype=a'),
('poster','https://movie.douban.com/subject/4864908/photos?type=R&start=60&sortby=like&size=a&subtype=a'),
('poster','https://movie.douban.com/subject/4864908/photos?type=R&start=90&sortby=like&size=a&subtype=a'),
('poster','https://movie.douban.com/subject/4864908/photos?type=R&start=120&sortby=like&size=a&subtype=a'),
('poster','https://movie.douban.com/subject/4864908/photos?type=R&start=150&sortby=like&size=a&subtype=a'),
('wallpaper','https://movie.douban.com/subject/4864908/photos?type=W&start=0&sortby=like&size=a&subtype=a'),
]
def get_image_list(page_links):
    image_url_info = []
    for page_link in page_links:
        print(page_link)
        soap = Common.getContextAsSelenium(page_link[1])
        ul_soup = soap.find('ul', {'class': 'poster-col3 clearfix'})
        images = ul_soup.find_all('li')
        for image in images:
            image_url = image.find('img')['src'].replace('/m/', '/1/')
            print((page_link[0], image_url))
            image_url_info.append((page_link[0], image_url))
    # print(image_url_info)
    for image_url in image_url_info:
        print("('" + image_url[0] + "','" + image_url[1] + "'),")
    return image_url_info


if __name__ == '__main__':

    # Step1: Get image page links
    # links = get_all_pages('4864908',51 , 6,1)

    # Step2: Get image links
    # get_image_list(page_links)

    # Step3: Image Download
    image_download(imagelinks,'影','4864908')

    # Combined Step1 to Step3 in one piece, but it always interrupted by anti-crawler
    # optimized_pic_url(links,'007之俄罗斯之恋')